{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"name":"CH05a_BERT_fine-tuning.ipynb","provenance":[],"collapsed_sections":[],"machine_shape":"hm"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.5"},"widgets":{"application/vnd.jupyter.widget-state+json":{"039e6432a6f14183917059d202f09771":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9458b25457d7479ea482db9644985ef3","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a13ed5b6ac594fd1b3a8014b55d6f2eb","IPY_MODEL_688789419e74414ebabbeae1f2cd894d"]}},"9458b25457d7479ea482db9644985ef3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a13ed5b6ac594fd1b3a8014b55d6f2eb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_b385f33637f2441c8cb43d3a49d6e95c","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":231508,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":231508,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_03e8d968300a4b6aa6fe20758acc76df"}},"688789419e74414ebabbeae1f2cd894d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_fdc7029f70f549abadfd1c5a8d93aa88","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 232k/232k [00:21&lt;00:00, 10.8kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_55f7e6b46f934a2d8baead6bc7d8a7c9"}},"b385f33637f2441c8cb43d3a49d6e95c":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"03e8d968300a4b6aa6fe20758acc76df":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"fdc7029f70f549abadfd1c5a8d93aa88":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"55f7e6b46f934a2d8baead6bc7d8a7c9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"231bf3f90c7d4089b0362b2abacab3d7":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_79e8ece3481148f8b2f9f150fa115c11","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_f92c6ce909a84f6a9002f653c8f314a4","IPY_MODEL_bb96e9bea3f845aa8df3d2f13e17b3e4"]}},"79e8ece3481148f8b2f9f150fa115c11":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f92c6ce909a84f6a9002f653c8f314a4":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_e6f7d42c511f477ea3af02ffd1fa6f58","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":466062,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":466062,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c04b8a6379c3467ca66e58856867ff8b"}},"bb96e9bea3f845aa8df3d2f13e17b3e4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_665ea65db1bf4374a94754e2e3b94083","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 466k/466k [00:03&lt;00:00, 133kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_597a22ba95b142a8afc11a5fad0de80a"}},"e6f7d42c511f477ea3af02ffd1fa6f58":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c04b8a6379c3467ca66e58856867ff8b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"665ea65db1bf4374a94754e2e3b94083":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"597a22ba95b142a8afc11a5fad0de80a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"78655aac76e44ba38d83e094b4019388":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_0a5acdd657f945e4ac9354ee49afb3c9","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_ae4299529b2b449b8643bed8b29665b1","IPY_MODEL_fab3824377c14982800209b85feb0c0a"]}},"0a5acdd657f945e4ac9354ee49afb3c9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ae4299529b2b449b8643bed8b29665b1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_a11513d538b54eee8723807995360c5c","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":28,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":28,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d2853e13946a453c957d5de4f07e89a1"}},"fab3824377c14982800209b85feb0c0a":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f68387dbb24d4ff4b396733befdfce74","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 28.0/28.0 [00:16&lt;00:00, 1.73B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_8c88927297f54bc2a24675fa7b05b9d1"}},"a11513d538b54eee8723807995360c5c":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"d2853e13946a453c957d5de4f07e89a1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f68387dbb24d4ff4b396733befdfce74":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"8c88927297f54bc2a24675fa7b05b9d1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"1f05a40b2b66402ea0469ad8c7329cc7":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_6ef5712b066d4de18b014116337234d2","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_bf87bb8973fe48bda54f0ba292cce4d1","IPY_MODEL_4b1ea0a99b71494f92466bfbefcb4fe6"]}},"6ef5712b066d4de18b014116337234d2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"bf87bb8973fe48bda54f0ba292cce4d1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_2843a77e11804e2c93cb0b7982983692","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":442,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":442,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_caacd43bea0844b39e8bd88e5d8085b7"}},"4b1ea0a99b71494f92466bfbefcb4fe6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c252434ba91d49bbbaee718b40f66250","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 442/442 [00:01&lt;00:00, 404B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_95e35b64461143f481dd7027cfd7b125"}},"2843a77e11804e2c93cb0b7982983692":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"caacd43bea0844b39e8bd88e5d8085b7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c252434ba91d49bbbaee718b40f66250":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"95e35b64461143f481dd7027cfd7b125":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7fee82b99ba44d5a8ce3b1be0cb1e67a":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_16143a30701d419785c365b9110e0213","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_47036678a52646f5a7d51fbbe52c275b","IPY_MODEL_d49e4eaaf787438d8e94303d940a635d"]}},"16143a30701d419785c365b9110e0213":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"47036678a52646f5a7d51fbbe52c275b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_7e01798832e6483f905229c95cb5b984","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":267967963,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":267967963,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_e7ecfe77151044209de642ef13489c10"}},"d49e4eaaf787438d8e94303d940a635d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_033f6cbad05b41e5890237626865a361","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 268M/268M [00:13&lt;00:00, 19.7MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2de3d099bc3549ea8bd3e6a352cecac5"}},"7e01798832e6483f905229c95cb5b984":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"e7ecfe77151044209de642ef13489c10":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"033f6cbad05b41e5890237626865a361":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2de3d099bc3549ea8bd3e6a352cecac5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f74fbff04dbc4ba3bea44a01c2070f16":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_c28dfdc1331a42028affcfb35b8f5d54","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_6e946f63cebe4bae8dfa48fe3a1c4828","IPY_MODEL_bb13fa465e384786b16c11e1b2ba74d3"]}},"c28dfdc1331a42028affcfb35b8f5d54":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6e946f63cebe4bae8dfa48fe3a1c4828":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_1231f5fb21ba49ddaad5eee253943849","_dom_classes":[],"description":"Downloading: ","_model_name":"FloatProgressModel","bar_style":"success","max":1916,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1916,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f398d3ba3bf14d40b90304512ca6fc3e"}},"bb13fa465e384786b16c11e1b2ba74d3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0e23aaec2bb349d3a61efda6746ec982","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 4.75k/? [00:10&lt;00:00, 448B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_6dcd7f3d0d8a403bb5fdb66306db6e73"}},"1231f5fb21ba49ddaad5eee253943849":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"f398d3ba3bf14d40b90304512ca6fc3e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0e23aaec2bb349d3a61efda6746ec982":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"6dcd7f3d0d8a403bb5fdb66306db6e73":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"165b2f8e1116454fb29990cc960a4bbb":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_fdd311179284450385f39500cbff9bbb","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_b3dd91137bb9443a8c1ea6267dbd1d80","IPY_MODEL_be0652b3362649deb7f0fb3c1f39f2a7"]}},"fdd311179284450385f39500cbff9bbb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b3dd91137bb9443a8c1ea6267dbd1d80":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_f17414dc21cd4f1881d10cf4cd48d967","_dom_classes":[],"description":"Downloading: ","_model_name":"FloatProgressModel","bar_style":"success","max":1054,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1054,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c4bd58c77ba747b1b06994f397d43ac1"}},"be0652b3362649deb7f0fb3c1f39f2a7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_545e2177f72040738de73a825baba068","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2.17k/? [00:00&lt;00:00, 38.6kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_92881d19dd464d3da8048f693c8515a5"}},"f17414dc21cd4f1881d10cf4cd48d967":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c4bd58c77ba747b1b06994f397d43ac1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"545e2177f72040738de73a825baba068":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"92881d19dd464d3da8048f693c8515a5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ec54c30b81f44954b0c60be88ffddf47":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_f1c8ba0dcf694076bb631029ff0868dc","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3bc0146974f84cf7823f47b243a7acc2","IPY_MODEL_ff9d864ead564c5fba8f642f67e8dca1"]}},"f1c8ba0dcf694076bb631029ff0868dc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3bc0146974f84cf7823f47b243a7acc2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_30bc9b9e5d4d49dfaf83a30d89787b18","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":84125825,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":84125825,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_17dcf5d75ce240e9ac550c51e61d0330"}},"ff9d864ead564c5fba8f642f67e8dca1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_5bb8f7f28c924dec844092cc1da86c72","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 84.1M/84.1M [00:09&lt;00:00, 9.08MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_938bf047d9c34d768bc85d8d47c22c68"}},"30bc9b9e5d4d49dfaf83a30d89787b18":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"17dcf5d75ce240e9ac550c51e61d0330":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"5bb8f7f28c924dec844092cc1da86c72":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"938bf047d9c34d768bc85d8d47c22c68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d9bcf0e073274fddb16f54574a5b1b48":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_830b67c906414058b07c169358cc59e8","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_61203bca01634586bdc83bb9682c0dd4","IPY_MODEL_d7bf0e56924e4dd39cde9914c53cb360"]}},"830b67c906414058b07c169358cc59e8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"61203bca01634586bdc83bb9682c0dd4":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_3432ac012eb14fc6b035e63f4516ccec","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"info","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_167f812811124900a82c083466806af4"}},"d7bf0e56924e4dd39cde9914c53cb360":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0796587a3c6145978aeec36b3da68d48","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 25000/0 [00:04&lt;00:00, 1201.73 examples/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a5b903b09325491eb794940b45b072a6"}},"3432ac012eb14fc6b035e63f4516ccec":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"167f812811124900a82c083466806af4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0796587a3c6145978aeec36b3da68d48":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a5b903b09325491eb794940b45b072a6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b80d5cd8470a439daedcfcff99569506":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_17ce4ed76dfa418f9f17b0600e785329","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_88d6a06d6fdd4f3bbcf1d4e9983f16ac","IPY_MODEL_3751f71b92594cf9a022fc132ba0f7c5"]}},"17ce4ed76dfa418f9f17b0600e785329":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"88d6a06d6fdd4f3bbcf1d4e9983f16ac":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d4d0181731614882b51efb401bcf3650","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"info","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d5e8825f738b4f81b9af5d0b18544620"}},"3751f71b92594cf9a022fc132ba0f7c5":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_f07e0adc97bd42f6ab446a1fef9b9917","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 25000/0 [00:02&lt;00:00, 11423.08 examples/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_98616f7738a94e62b41db301610f89a0"}},"d4d0181731614882b51efb401bcf3650":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"d5e8825f738b4f81b9af5d0b18544620":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f07e0adc97bd42f6ab446a1fef9b9917":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"98616f7738a94e62b41db301610f89a0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"14cbabd64d524dd2bf3ffc4f4a121d69":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_ab828dd3fff8463da5fb511e253a4acf","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_98febe9bd82245f3abe9d121a3c0914b","IPY_MODEL_f5190dc768e6487099e1ad2896b3ff8c"]}},"ab828dd3fff8463da5fb511e253a4acf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"98febe9bd82245f3abe9d121a3c0914b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_e7cba4a98e9a46bfbe7e506cbe1981ab","_dom_classes":[],"description":"","_model_name":"FloatProgressModel","bar_style":"info","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_bc2967f766304c62ad7dce0f44328c9f"}},"f5190dc768e6487099e1ad2896b3ff8c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_2bfd6c57c4e64d3c9750dea9fdbe9756","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 50000/0 [00:04&lt;00:00, 12268.39 examples/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_fdc65af0342646b1bc0475336e8bbfba"}},"e7cba4a98e9a46bfbe7e506cbe1981ab":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"bc2967f766304c62ad7dce0f44328c9f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2bfd6c57c4e64d3c9750dea9fdbe9756":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"fdc65af0342646b1bc0475336e8bbfba":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"dfbcb2c951134d5e98b9ccacdf0b09f5":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_fc3b3b4b0178445593712ab8b745919f","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_472e69e5750b4c3ca3ea89fda96fc5dc","IPY_MODEL_32cb03fdf6a34028bd7aa8d942a307c6"]}},"fc3b3b4b0178445593712ab8b745919f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"472e69e5750b4c3ca3ea89fda96fc5dc":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d03436c2ef924ee9878af419bf8936fb","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":4,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":4,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c44909e1ae244943bc91e939871df1e6"}},"32cb03fdf6a34028bd7aa8d942a307c6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_814943ac8f1640bdbd9a85be81ad70a4","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 4/4 [00:13&lt;00:00,  3.40s/ba]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_c66d391d055746d8b44c1c5cff8bed68"}},"d03436c2ef924ee9878af419bf8936fb":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"c44909e1ae244943bc91e939871df1e6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"814943ac8f1640bdbd9a85be81ad70a4":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"c66d391d055746d8b44c1c5cff8bed68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"1c1f60b5eff7409db937413e8bbbf4c2":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_50dda90fde9c4e1a80bdfe70944272b8","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_a163e110716b43b29a69997dfe5b20d7","IPY_MODEL_14d73fee82fa44499216e0a917c9897e"]}},"50dda90fde9c4e1a80bdfe70944272b8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"a163e110716b43b29a69997dfe5b20d7":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d0dee38ac5ef4329b3788d0ebdf846bd","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_4147df48a35344178a3cb6108208935a"}},"14d73fee82fa44499216e0a917c9897e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_7952bd9695a54d26b750b629a05215dd","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1/1 [00:00&lt;00:00,  1.10ba/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_5913cd0b1bf24cf58d045c0c7bc4d0f8"}},"d0dee38ac5ef4329b3788d0ebdf846bd":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"4147df48a35344178a3cb6108208935a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"7952bd9695a54d26b750b629a05215dd":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"5913cd0b1bf24cf58d045c0c7bc4d0f8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"20834d9e5f5e483186bc6a1f110ff67c":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_409226d70af94b5d9e4ad02601eb2974","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_b4d2020d434d44b78bc6d283058131bc","IPY_MODEL_9da82a3d83a44f6b8f4167d624190d8f"]}},"409226d70af94b5d9e4ad02601eb2974":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b4d2020d434d44b78bc6d283058131bc":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_14c54a79ec15437397f94257dd5c5d71","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":1,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_11a173333b3e444b91d80338cd87c0ff"}},"9da82a3d83a44f6b8f4167d624190d8f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d2fe41b819fb434aa21b79d72e45c12d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1/1 [00:11&lt;00:00, 11.19s/ba]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1fa1568a5ff747a6b2ff4726d49ae632"}},"14c54a79ec15437397f94257dd5c5d71":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"11a173333b3e444b91d80338cd87c0ff":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d2fe41b819fb434aa21b79d72e45c12d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"1fa1568a5ff747a6b2ff4726d49ae632":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"Xr50xchZunFD"},"source":["# Fine-tuning BERT model for Binary Text Classification"]},{"cell_type":"markdown","metadata":{"id":"tszbPHwjFfkP"},"source":["## Preparation for Google Collab"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BRzRpdbLDDKv","executionInfo":{"status":"ok","timestamp":1625395992357,"user_tz":-180,"elapsed":28655,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"c1071feb-2e6f-4b52-ff96-02e28efc1b61"},"source":["import torch, os\n","from google.colab import drive\n","drive.mount('/content/drive')\n","os.listdir()\n","os.chdir(\"drive/MyDrive/akademi/Packt NLP with Transformers/CH05\")"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4wW9BBcot81Q","executionInfo":{"status":"ok","timestamp":1625396004941,"user_tz":-180,"elapsed":10529,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"d69918bf-5840-4ad8-a563-ffe3ce22d50a"},"source":["!pip install transformers datasets"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Collecting transformers\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/1a/41c644c963249fd7f3836d926afa1e3f1cc234a1c40d80c5f03ad8f6f1b2/transformers-4.8.2-py3-none-any.whl (2.5MB)\n","\u001b[K     |████████████████████████████████| 2.5MB 4.2MB/s \n","\u001b[?25hCollecting datasets\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/a2/d4e1024c891506e1cee8f9d719d20831bac31cb5b7416983c4d2f65a6287/datasets-1.8.0-py3-none-any.whl (237kB)\n","\u001b[K     |████████████████████████████████| 245kB 17.7MB/s \n","\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.5.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n","Collecting huggingface-hub==0.0.12\n","  Downloading https://files.pythonhosted.org/packages/2f/ee/97e253668fda9b17e968b3f97b2f8e53aa0127e8807d24a547687423fe0b/huggingface_hub-0.0.12-py3-none-any.whl\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Collecting sacremoses\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n","\u001b[K     |████████████████████████████████| 901kB 23.4MB/s \n","\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n","\u001b[K     |████████████████████████████████| 3.3MB 33.5MB/s \n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n","Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n","Collecting fsspec\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/3a/666e63625a19883ae8e1674099e631f9737bd5478c4790e5ad49c5ac5261/fsspec-2021.6.1-py3-none-any.whl (115kB)\n","\u001b[K     |████████████████████████████████| 122kB 47.6MB/s \n","\u001b[?25hRequirement already satisfied: pyarrow<4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n","Collecting xxhash\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/4f/0a862cad26aa2ed7a7cd87178cbbfa824fc1383e472d63596a0d018374e7/xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n","\u001b[K     |████████████████████████████████| 245kB 32.5MB/s \n","\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n","Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n","Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n","Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n","Installing collected packages: huggingface-hub, sacremoses, tokenizers, transformers, fsspec, xxhash, datasets\n","Successfully installed datasets-1.8.0 fsspec-2021.6.1 huggingface-hub-0.0.12 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.8.2 xxhash-2.0.2\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"zRXnOvJYWtJS","executionInfo":{"status":"ok","timestamp":1625396009817,"user_tz":-180,"elapsed":614,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from torch import cuda\n","device = 'cuda' if cuda.is_available() else 'cpu'"],"execution_count":3,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"BZv1tmcx_pNo"},"source":["### Loading pre-trained model and tokenizer"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":367,"referenced_widgets":["039e6432a6f14183917059d202f09771","9458b25457d7479ea482db9644985ef3","a13ed5b6ac594fd1b3a8014b55d6f2eb","688789419e74414ebabbeae1f2cd894d","b385f33637f2441c8cb43d3a49d6e95c","03e8d968300a4b6aa6fe20758acc76df","fdc7029f70f549abadfd1c5a8d93aa88","55f7e6b46f934a2d8baead6bc7d8a7c9","231bf3f90c7d4089b0362b2abacab3d7","79e8ece3481148f8b2f9f150fa115c11","f92c6ce909a84f6a9002f653c8f314a4","bb96e9bea3f845aa8df3d2f13e17b3e4","e6f7d42c511f477ea3af02ffd1fa6f58","c04b8a6379c3467ca66e58856867ff8b","665ea65db1bf4374a94754e2e3b94083","597a22ba95b142a8afc11a5fad0de80a","78655aac76e44ba38d83e094b4019388","0a5acdd657f945e4ac9354ee49afb3c9","ae4299529b2b449b8643bed8b29665b1","fab3824377c14982800209b85feb0c0a","a11513d538b54eee8723807995360c5c","d2853e13946a453c957d5de4f07e89a1","f68387dbb24d4ff4b396733befdfce74","8c88927297f54bc2a24675fa7b05b9d1","1f05a40b2b66402ea0469ad8c7329cc7","6ef5712b066d4de18b014116337234d2","bf87bb8973fe48bda54f0ba292cce4d1","4b1ea0a99b71494f92466bfbefcb4fe6","2843a77e11804e2c93cb0b7982983692","caacd43bea0844b39e8bd88e5d8085b7","c252434ba91d49bbbaee718b40f66250","95e35b64461143f481dd7027cfd7b125","7fee82b99ba44d5a8ce3b1be0cb1e67a","16143a30701d419785c365b9110e0213","47036678a52646f5a7d51fbbe52c275b","d49e4eaaf787438d8e94303d940a635d","7e01798832e6483f905229c95cb5b984","e7ecfe77151044209de642ef13489c10","033f6cbad05b41e5890237626865a361","2de3d099bc3549ea8bd3e6a352cecac5"]},"id":"DE1Rwh2BtnwT","executionInfo":{"status":"ok","timestamp":1625396029653,"user_tz":-180,"elapsed":17573,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"a38ff2f2-44f7-4a94-fb66-5a38b59d38d5"},"source":["from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n","model_path= 'distilbert-base-uncased'\n","tokenizer = DistilBertTokenizerFast.from_pretrained(model_path)\n","model = DistilBertForSequenceClassification.from_pretrained(model_path, id2label={0:\"NEG\", 1:\"POS\"}, label2id={\"NEG\":0, \"POS\":1})"],"execution_count":4,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"039e6432a6f14183917059d202f09771","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"231bf3f90c7d4089b0362b2abacab3d7","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"78655aac76e44ba38d83e094b4019388","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1f05a40b2b66402ea0469ad8c7329cc7","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"7fee82b99ba44d5a8ce3b1be0cb1e67a","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=267967963.0, style=ProgressStyle(descri…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"stream","text":["Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_transform.bias', 'vocab_projector.bias', 'vocab_layer_norm.bias']\n","- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias', 'pre_classifier.weight', 'pre_classifier.bias']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"licNE7yQS8uw"},"source":["## Loading popular IMDB dataset"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":252,"referenced_widgets":["f74fbff04dbc4ba3bea44a01c2070f16","c28dfdc1331a42028affcfb35b8f5d54","6e946f63cebe4bae8dfa48fe3a1c4828","bb13fa465e384786b16c11e1b2ba74d3","1231f5fb21ba49ddaad5eee253943849","f398d3ba3bf14d40b90304512ca6fc3e","0e23aaec2bb349d3a61efda6746ec982","6dcd7f3d0d8a403bb5fdb66306db6e73","165b2f8e1116454fb29990cc960a4bbb","fdd311179284450385f39500cbff9bbb","b3dd91137bb9443a8c1ea6267dbd1d80","be0652b3362649deb7f0fb3c1f39f2a7","f17414dc21cd4f1881d10cf4cd48d967","c4bd58c77ba747b1b06994f397d43ac1","545e2177f72040738de73a825baba068","92881d19dd464d3da8048f693c8515a5","ec54c30b81f44954b0c60be88ffddf47","f1c8ba0dcf694076bb631029ff0868dc","3bc0146974f84cf7823f47b243a7acc2","ff9d864ead564c5fba8f642f67e8dca1","30bc9b9e5d4d49dfaf83a30d89787b18","17dcf5d75ce240e9ac550c51e61d0330","5bb8f7f28c924dec844092cc1da86c72","938bf047d9c34d768bc85d8d47c22c68","d9bcf0e073274fddb16f54574a5b1b48","830b67c906414058b07c169358cc59e8","61203bca01634586bdc83bb9682c0dd4","d7bf0e56924e4dd39cde9914c53cb360","3432ac012eb14fc6b035e63f4516ccec","167f812811124900a82c083466806af4","0796587a3c6145978aeec36b3da68d48","a5b903b09325491eb794940b45b072a6","b80d5cd8470a439daedcfcff99569506","17ce4ed76dfa418f9f17b0600e785329","88d6a06d6fdd4f3bbcf1d4e9983f16ac","3751f71b92594cf9a022fc132ba0f7c5","d4d0181731614882b51efb401bcf3650","d5e8825f738b4f81b9af5d0b18544620","f07e0adc97bd42f6ab446a1fef9b9917","98616f7738a94e62b41db301610f89a0","14cbabd64d524dd2bf3ffc4f4a121d69","ab828dd3fff8463da5fb511e253a4acf","98febe9bd82245f3abe9d121a3c0914b","f5190dc768e6487099e1ad2896b3ff8c","e7cba4a98e9a46bfbe7e506cbe1981ab","bc2967f766304c62ad7dce0f44328c9f","2bfd6c57c4e64d3c9750dea9fdbe9756","fdc65af0342646b1bc0475336e8bbfba"]},"id":"w2HdjLQEXTos","executionInfo":{"status":"ok","timestamp":1625396090786,"user_tz":-180,"elapsed":56296,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"cc00a09e-bcbc-4dbd-d6ba-ffa3965436b3"},"source":["from datasets import load_dataset\n","\n","# to take entire dataset from original train 25 K AND TEST 25K\n","'''\n","imdb_train= load_dataset('imdb', split=\"train\")\n","imdb_test= load_dataset('imdb', split=\"test[:6250]+test[-6250:]\")\n","imdb_val= load_dataset('imdb', split=\"test[6250:12500]+test[-12500:-6250]\")\n","\n","'''\n","# to take smaller portion 4K for train, 1K for test and 1K for validation\n","imdb_train= load_dataset('imdb', split=\"train[:2000]+train[-2000:]\")\n","imdb_test= load_dataset('imdb', split=\"test[:500]+test[-500:]\")\n","imdb_val= load_dataset('imdb', split=\"test[500:1000]+test[-1000:-500]\")\n"],"execution_count":5,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f74fbff04dbc4ba3bea44a01c2070f16","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1916.0, style=ProgressStyle(description…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"165b2f8e1116454fb29990cc960a4bbb","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1054.0, style=ProgressStyle(description…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","Downloading and preparing dataset imdb/plain_text (download: 80.23 MiB, generated: 127.02 MiB, post-processed: Unknown size, total: 207.25 MiB) to /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/e3c66f1788a67a89c7058d97ff62b6c30531e05b549de56d3ab91891f0561f9a...\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ec54c30b81f44954b0c60be88ffddf47","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=84125825.0, style=ProgressStyle(descrip…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"d9bcf0e073274fddb16f54574a5b1b48","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b80d5cd8470a439daedcfcff99569506","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\r"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"14cbabd64d524dd2bf3ffc4f4a121d69","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\rDataset imdb downloaded and prepared to /root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/e3c66f1788a67a89c7058d97ff62b6c30531e05b549de56d3ab91891f0561f9a. Subsequent calls will reuse this data.\n"],"name":"stdout"},{"output_type":"stream","text":["Reusing dataset imdb (/root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/e3c66f1788a67a89c7058d97ff62b6c30531e05b549de56d3ab91891f0561f9a)\n","Reusing dataset imdb (/root/.cache/huggingface/datasets/imdb/plain_text/1.0.0/e3c66f1788a67a89c7058d97ff62b6c30531e05b549de56d3ab91891f0561f9a)\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_kfFp7nkbki4","executionInfo":{"status":"ok","timestamp":1625396090787,"user_tz":-180,"elapsed":22,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"05acc959-6c55-4918-e9cc-bde120f8802f"},"source":["imdb_train.shape, imdb_test.shape, imdb_val.shape"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/plain":["((4000, 2), (1000, 2), (1000, 2))"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":164,"referenced_widgets":["dfbcb2c951134d5e98b9ccacdf0b09f5","fc3b3b4b0178445593712ab8b745919f","472e69e5750b4c3ca3ea89fda96fc5dc","32cb03fdf6a34028bd7aa8d942a307c6","d03436c2ef924ee9878af419bf8936fb","c44909e1ae244943bc91e939871df1e6","814943ac8f1640bdbd9a85be81ad70a4","c66d391d055746d8b44c1c5cff8bed68","1c1f60b5eff7409db937413e8bbbf4c2","50dda90fde9c4e1a80bdfe70944272b8","a163e110716b43b29a69997dfe5b20d7","14d73fee82fa44499216e0a917c9897e","d0dee38ac5ef4329b3788d0ebdf846bd","4147df48a35344178a3cb6108208935a","7952bd9695a54d26b750b629a05215dd","5913cd0b1bf24cf58d045c0c7bc4d0f8","20834d9e5f5e483186bc6a1f110ff67c","409226d70af94b5d9e4ad02601eb2974","b4d2020d434d44b78bc6d283058131bc","9da82a3d83a44f6b8f4167d624190d8f","14c54a79ec15437397f94257dd5c5d71","11a173333b3e444b91d80338cd87c0ff","d2fe41b819fb434aa21b79d72e45c12d","1fa1568a5ff747a6b2ff4726d49ae632"]},"id":"WpuCsmUVXgm9","executionInfo":{"status":"ok","timestamp":1625396100497,"user_tz":-180,"elapsed":3345,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"cfd96b57-1f5c-41bb-c79a-e046973dcdd9"},"source":["enc_train = imdb_train.map(lambda e: tokenizer( e['text'], padding=True, truncation=True), batched=True, batch_size=1000) \n","enc_test =  imdb_test.map(lambda e: tokenizer( e['text'], padding=True, truncation=True), batched=True, batch_size=1000) \n","enc_val =   imdb_val.map(lambda e: tokenizer( e['text'], padding=True, truncation=True), batched=True, batch_size=1000) "],"execution_count":8,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"dfbcb2c951134d5e98b9ccacdf0b09f5","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1c1f60b5eff7409db937413e8bbbf4c2","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"20834d9e5f5e483186bc6a1f110ff67c","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":419},"id":"-4lY_LHTXgpa","executionInfo":{"status":"ok","timestamp":1625396107583,"user_tz":-180,"elapsed":2603,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"79392180-6ca4-4d7b-fbd6-d8f46e550976"},"source":["import pandas as pd\n","pd.DataFrame(enc_train)"],"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>attention_mask</th>\n","      <th>input_ids</th>\n","      <th>label</th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 22953, 2213, 4381, 2152, 2003, 1037, 947...</td>\n","      <td>1</td>\n","      <td>Bromwell High is a cartoon comedy. It ran at t...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 11573, 2791, 1006, 2030, 2160, 24913, 20...</td>\n","      <td>1</td>\n","      <td>Homelessness (or Houselessness as George Carli...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 8235, 2058, 1011, 3772, 2011, 23920, 575...</td>\n","      <td>1</td>\n","      <td>Brilliant over-acting by Lesley Ann Warren. Be...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 2023, 2003, 4089, 1996, 2087, 2104, 9250...</td>\n","      <td>1</td>\n","      <td>This is easily the most underrated film inn th...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 2023, 2003, 2025, 1996, 5171, 11463, 837...</td>\n","      <td>1</td>\n","      <td>This is not the typical Mel Brooks film. It wa...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>3995</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 2875, 1996, 2203, 1997, 1996, 3185, 1010...</td>\n","      <td>0</td>\n","      <td>Towards the end of the movie, I felt it was to...</td>\n","    </tr>\n","    <tr>\n","      <th>3996</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 2023, 2003, 1996, 2785, 1997, 3185, 2008...</td>\n","      <td>0</td>\n","      <td>This is the kind of movie that my enemies cont...</td>\n","    </tr>\n","    <tr>\n","      <th>3997</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 1045, 2387, 1005, 6934, 1005, 2197, 2305...</td>\n","      <td>0</td>\n","      <td>I saw 'Descent' last night at the Stockholm Fi...</td>\n","    </tr>\n","    <tr>\n","      <th>3998</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 2070, 3152, 2008, 2017, 4060, 2039, 2005...</td>\n","      <td>0</td>\n","      <td>Some films that you pick up for a pound turn o...</td>\n","    </tr>\n","    <tr>\n","      <th>3999</th>\n","      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n","      <td>[101, 2023, 2003, 2028, 1997, 1996, 12873, 435...</td>\n","      <td>0</td>\n","      <td>This is one of the dumbest films, I've ever se...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4000 rows × 4 columns</p>\n","</div>"],"text/plain":["                                         attention_mask  ...                                               text\n","0     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  Bromwell High is a cartoon comedy. It ran at t...\n","1     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  Homelessness (or Houselessness as George Carli...\n","2     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  Brilliant over-acting by Lesley Ann Warren. Be...\n","3     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  This is easily the most underrated film inn th...\n","4     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  This is not the typical Mel Brooks film. It wa...\n","...                                                 ...  ...                                                ...\n","3995  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  Towards the end of the movie, I felt it was to...\n","3996  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  This is the kind of movie that my enemies cont...\n","3997  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  I saw 'Descent' last night at the Stockholm Fi...\n","3998  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  Some films that you pick up for a pound turn o...\n","3999  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...  ...  This is one of the dumbest films, I've ever se...\n","\n","[4000 rows x 4 columns]"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"markdown","metadata":{"id":"ACNK9ZQKD4mt"},"source":["## Preparing training settings with TrainingArguments and Trainer class\n","\n","TrainingArguments is the subset of the arguments we use in our example scripts **which relate to the training loop\n","itself**."]},{"cell_type":"code","metadata":{"id":"OZD1jnhFWEd5","executionInfo":{"status":"ok","timestamp":1625396111857,"user_tz":-180,"elapsed":1541,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from transformers import TrainingArguments, Trainer\n","training_args = TrainingArguments(\n","    # The output directory where the model predictions and checkpoints will be written\n","    output_dir='./MyIMDBModel', \n","    do_train=True,\n","    do_eval=True,\n","    #  The number of epochs, defaults to 3.0 \n","    num_train_epochs=3,              \n","    per_device_train_batch_size=32,  \n","    per_device_eval_batch_size=64,\n","    # Number of steps used for a linear warmup\n","    warmup_steps=100,                \n","    weight_decay=0.01,\n","    logging_strategy='steps', \n","    # TensorBoard log directory               \n","    logging_dir='./logs',            \n","    logging_steps=50,\n","    # other options : no, steps\n","    evaluation_strategy=\"steps\",\n","    save_strategy=\"epoch\",\n","    fp16=cuda.is_available(),\n","    load_best_model_at_end=True\n",")"],"execution_count":10,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"HUn5uFHkbPEb"},"source":["Let's design our evaluation metrics as follows:"]},{"cell_type":"code","metadata":{"id":"8Ct3v6RZZSmj","executionInfo":{"status":"ok","timestamp":1625396117168,"user_tz":-180,"elapsed":6,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n","def compute_metrics(pred):\n","    labels = pred.label_ids\n","    preds = pred.predictions.argmax(-1)\n","    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')\n","    acc = accuracy_score(labels, preds)\n","    return {\n","        'Accuracy': acc,\n","        'F1': f1,\n","        'Precision': precision,\n","        'Recall': recall\n","    }"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"ztS1YQcqWEgZ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625396126034,"user_tz":-180,"elapsed":6920,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"4cc11ff8-39d9-4cd1-d399-1a86eac460bd"},"source":["trainer = Trainer(\n","    # the pre-trained model that will be fine-tuned \n","    model=model,\n","     # training arguments that we defined above                        \n","    args=training_args,\n","    # training and validation dataset                 \n","    train_dataset=enc_train,         \n","    eval_dataset=enc_val,            \n","    compute_metrics= compute_metrics\n",")"],"execution_count":12,"outputs":[{"output_type":"stream","text":["Using amp fp16 backend\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"5DfXPD8Lznss","executionInfo":{"status":"ok","timestamp":1625396129126,"user_tz":-180,"elapsed":597,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":[""],"execution_count":12,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"x354spCgdVEX"},"source":["Trainer class is an optimized tool for Transformer to organize training and evalulation process for PyTorch and Tensorflow."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"zF7FUYOzWFFT","executionInfo":{"status":"ok","timestamp":1625396735577,"user_tz":-180,"elapsed":604576,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"8f823448-142e-4720-9b19-d9b1a0ba6676"},"source":["results=trainer.train()"],"execution_count":13,"outputs":[{"output_type":"stream","text":["The following columns in the training set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running training *****\n","  Num examples = 4000\n","  Num Epochs = 3\n","  Instantaneous batch size per device = 32\n","  Total train batch size (w. parallel, distributed & accumulation) = 32\n","  Gradient Accumulation steps = 1\n","  Total optimization steps = 375\n"],"name":"stderr"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='375' max='375' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [375/375 10:03, Epoch 3/3]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","      <th>Validation Loss</th>\n","      <th>Accuracy</th>\n","      <th>F1</th>\n","      <th>Precision</th>\n","      <th>Recall</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <td>50</td>\n","      <td>0.653400</td>\n","      <td>0.445251</td>\n","      <td>0.824000</td>\n","      <td>0.821341</td>\n","      <td>0.844511</td>\n","      <td>0.824000</td>\n","    </tr>\n","    <tr>\n","      <td>100</td>\n","      <td>0.362200</td>\n","      <td>0.256781</td>\n","      <td>0.907000</td>\n","      <td>0.906992</td>\n","      <td>0.907132</td>\n","      <td>0.907000</td>\n","    </tr>\n","    <tr>\n","      <td>150</td>\n","      <td>0.293500</td>\n","      <td>0.377014</td>\n","      <td>0.851000</td>\n","      <td>0.848786</td>\n","      <td>0.872835</td>\n","      <td>0.851000</td>\n","    </tr>\n","    <tr>\n","      <td>200</td>\n","      <td>0.224300</td>\n","      <td>0.262217</td>\n","      <td>0.906000</td>\n","      <td>0.905976</td>\n","      <td>0.906416</td>\n","      <td>0.906000</td>\n","    </tr>\n","    <tr>\n","      <td>250</td>\n","      <td>0.165300</td>\n","      <td>0.232591</td>\n","      <td>0.917000</td>\n","      <td>0.916976</td>\n","      <td>0.917483</td>\n","      <td>0.917000</td>\n","    </tr>\n","    <tr>\n","      <td>300</td>\n","      <td>0.076100</td>\n","      <td>0.301682</td>\n","      <td>0.917000</td>\n","      <td>0.916996</td>\n","      <td>0.917082</td>\n","      <td>0.917000</td>\n","    </tr>\n","    <tr>\n","      <td>350</td>\n","      <td>0.093500</td>\n","      <td>0.298886</td>\n","      <td>0.918000</td>\n","      <td>0.917984</td>\n","      <td>0.918328</td>\n","      <td>0.918000</td>\n","    </tr>\n","  </tbody>\n","</table><p>"],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-50\n","Configuration saved in ./MyIMDBModel/checkpoint-50/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-50/pytorch_model.bin\n","/usr/local/lib/python3.7/dist-packages/transformers/trainer.py:1299: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n","  args.max_grad_norm,\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-100\n","Configuration saved in ./MyIMDBModel/checkpoint-100/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-100/pytorch_model.bin\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-125\n","Configuration saved in ./MyIMDBModel/checkpoint-125/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-125/pytorch_model.bin\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-150\n","Configuration saved in ./MyIMDBModel/checkpoint-150/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-150/pytorch_model.bin\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-200\n","Configuration saved in ./MyIMDBModel/checkpoint-200/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-200/pytorch_model.bin\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-250\n","Configuration saved in ./MyIMDBModel/checkpoint-250/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-250/pytorch_model.bin\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-250\n","Configuration saved in ./MyIMDBModel/checkpoint-250/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-250/pytorch_model.bin\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-300\n","Configuration saved in ./MyIMDBModel/checkpoint-300/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-300/pytorch_model.bin\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-350\n","Configuration saved in ./MyIMDBModel/checkpoint-350/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-350/pytorch_model.bin\n","Saving model checkpoint to ./MyIMDBModel/checkpoint-375\n","Configuration saved in ./MyIMDBModel/checkpoint-375/config.json\n","Model weights saved in ./MyIMDBModel/checkpoint-375/pytorch_model.bin\n","\n","\n","Training completed. Do not forget to share your model on huggingface.co/models =)\n","\n","\n","Loading best model from ./MyIMDBModel/checkpoint-250 (score: 0.23259127140045166).\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"w_LLlhK5i69Q"},"source":["* For small portion\n","\n","We have 4K training examples and the batch size is 32, which means each epoch takes 4k/32 = 125 steps. For 3 epoc, this will have 375 steps as wee see in the output.\n","\n","* For full dataset\n","\n","We have 25K training  examples with the batch size pf 32, so we will have 25K/32 ~=782 (roughly) steps. for 3 epoch we will have 2346 steps to run as seen in the progress bar "]},{"cell_type":"markdown","metadata":{"id":"bWmDkDVWIzzO"},"source":["Trainer model keeps the best model at the end. Lets evaluate the model on the train/test/validation"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":366},"id":"MfBHiik5XMmf","executionInfo":{"status":"ok","timestamp":1625396803684,"user_tz":-180,"elapsed":53768,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"b61af1f8-3696-4462-aaf3-a3b7d4479081"},"source":["q=[trainer.evaluate(eval_dataset=data) for data in [enc_train, enc_val, enc_test]]\n","pd.DataFrame(q, index=[\"train\",\"val\",\"test\"]).iloc[:,:5]"],"execution_count":14,"outputs":[{"output_type":"stream","text":["The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 4000\n","  Batch size = 64\n"],"name":"stderr"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='95' max='63' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [63/63 00:53]\n","    </div>\n","    "],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n","The following columns in the evaluation set  don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text.\n","***** Running Evaluation *****\n","  Num examples = 1000\n","  Batch size = 64\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>eval_loss</th>\n","      <th>eval_Accuracy</th>\n","      <th>eval_F1</th>\n","      <th>eval_Precision</th>\n","      <th>eval_Recall</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>train</th>\n","      <td>0.077374</td>\n","      <td>0.97975</td>\n","      <td>0.979750</td>\n","      <td>0.979751</td>\n","      <td>0.97975</td>\n","    </tr>\n","    <tr>\n","      <th>val</th>\n","      <td>0.232591</td>\n","      <td>0.91700</td>\n","      <td>0.916976</td>\n","      <td>0.917483</td>\n","      <td>0.91700</td>\n","    </tr>\n","    <tr>\n","      <th>test</th>\n","      <td>0.245722</td>\n","      <td>0.90400</td>\n","      <td>0.904000</td>\n","      <td>0.904006</td>\n","      <td>0.90400</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["       eval_loss  eval_Accuracy   eval_F1  eval_Precision  eval_Recall\n","train   0.077374        0.97975  0.979750        0.979751      0.97975\n","val     0.232591        0.91700  0.916976        0.917483      0.91700\n","test    0.245722        0.90400  0.904000        0.904006      0.90400"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vQRdl3cfc53K","executionInfo":{"status":"ok","timestamp":1625396818121,"user_tz":-180,"elapsed":8710,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"e1f9e51c-71fe-4c19-aeef-1c419e86b627"},"source":["# saving the best fine-tuned model & tokenizer\n","model_save_path = \"MyBestIMDBModel\"\n","trainer.save_model(model_save_path)\n","tokenizer.save_pretrained(model_save_path)"],"execution_count":15,"outputs":[{"output_type":"stream","text":["Saving model checkpoint to MyBestIMDBModel\n","Configuration saved in MyBestIMDBModel/config.json\n","Model weights saved in MyBestIMDBModel/pytorch_model.bin\n","tokenizer config file saved in MyBestIMDBModel/tokenizer_config.json\n","Special tokens file saved in MyBestIMDBModel/special_tokens_map.json\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["('MyBestIMDBModel/tokenizer_config.json',\n"," 'MyBestIMDBModel/special_tokens_map.json',\n"," 'MyBestIMDBModel/vocab.txt',\n"," 'MyBestIMDBModel/added_tokens.json',\n"," 'MyBestIMDBModel/tokenizer.json')"]},"metadata":{"tags":[]},"execution_count":15}]},{"cell_type":"code","metadata":{"id":"D2a3rNrLfHQ3","executionInfo":{"status":"ok","timestamp":1625396821293,"user_tz":-180,"elapsed":414,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}}},"source":["def get_prediction(text):\n","    inputs = tokenizer(text, padding=True, truncation=True, max_length=250, return_tensors=\"pt\").to(device)\n","    outputs = model(inputs[\"input_ids\"].to(device),inputs[\"attention_mask\"].to(device))\n","    probs = outputs[0].softmax(1)\n","    return probs, probs.argmax()"],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3nSJDJ5mfTdu","executionInfo":{"status":"ok","timestamp":1625396830460,"user_tz":-180,"elapsed":547,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"264aa193-f7c5-4e3b-c5a6-2c6f8cf3b0fb"},"source":["model.to(device)\n","text = \"I didn't like the movie since it bored me \"\n","get_prediction(text)[1].item()"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"code","metadata":{"id":"rEPQzT4UDk7B"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ze4nteT-mCUC"},"source":["Use the model with pipeline "]},{"cell_type":"code","metadata":{"id":"iQ7sVE4blgzX","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625396836461,"user_tz":-180,"elapsed":1851,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"3cfe2166-d20d-4117-bf76-d85e645e0e3c"},"source":["from transformers import pipeline, DistilBertForSequenceClassification, DistilBertTokenizerFast\n","model = DistilBertForSequenceClassification.from_pretrained(\"MyBestIMDBModel\")\n","tokenizer= DistilBertTokenizerFast.from_pretrained(\"MyBestIMDBModel\")\n","nlp= pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer)\n","nlp(\"the movie was very impressive\")"],"execution_count":19,"outputs":[{"output_type":"stream","text":["loading configuration file MyBestIMDBModel/config.json\n","Model config DistilBertConfig {\n","  \"_name_or_path\": \"distilbert-base-uncased\",\n","  \"activation\": \"gelu\",\n","  \"architectures\": [\n","    \"DistilBertForSequenceClassification\"\n","  ],\n","  \"attention_dropout\": 0.1,\n","  \"dim\": 768,\n","  \"dropout\": 0.1,\n","  \"hidden_dim\": 3072,\n","  \"id2label\": {\n","    \"0\": \"NEG\",\n","    \"1\": \"POS\"\n","  },\n","  \"initializer_range\": 0.02,\n","  \"label2id\": {\n","    \"NEG\": 0,\n","    \"POS\": 1\n","  },\n","  \"max_position_embeddings\": 512,\n","  \"model_type\": \"distilbert\",\n","  \"n_heads\": 12,\n","  \"n_layers\": 6,\n","  \"pad_token_id\": 0,\n","  \"problem_type\": \"single_label_classification\",\n","  \"qa_dropout\": 0.1,\n","  \"seq_classif_dropout\": 0.2,\n","  \"sinusoidal_pos_embds\": false,\n","  \"tie_weights_\": true,\n","  \"transformers_version\": \"4.8.2\",\n","  \"vocab_size\": 30522\n","}\n","\n","loading weights file MyBestIMDBModel/pytorch_model.bin\n","All model checkpoint weights were used when initializing DistilBertForSequenceClassification.\n","\n","All the weights of DistilBertForSequenceClassification were initialized from the model checkpoint at MyBestIMDBModel.\n","If your task is similar to the task the model of the checkpoint was trained on, you can already use DistilBertForSequenceClassification for predictions without further training.\n","Didn't find file MyBestIMDBModel/added_tokens.json. We won't load it.\n","loading file MyBestIMDBModel/vocab.txt\n","loading file MyBestIMDBModel/tokenizer.json\n","loading file None\n","loading file MyBestIMDBModel/special_tokens_map.json\n","loading file MyBestIMDBModel/tokenizer_config.json\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["[{'label': 'POS', 'score': 0.9790748953819275}]"]},"metadata":{"tags":[]},"execution_count":19}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0Jv-X4pN3H9Y","executionInfo":{"status":"ok","timestamp":1625396843499,"user_tz":-180,"elapsed":441,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"}},"outputId":"8c455ed1-00ed-475d-9bcd-951960912d68"},"source":["nlp(\"the script of the picture was very poor\")"],"execution_count":20,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'label': 'NEG', 'score': 0.98270583152771}]"]},"metadata":{"tags":[]},"execution_count":20}]},{"cell_type":"code","metadata":{"id":"GPiDrENVJ42j"},"source":[""],"execution_count":null,"outputs":[]}]}